﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Web;

namespace FetchAIP
{
    class Crawl
    {
        // Create a new cookie container
        CookieContainer cookies = new CookieContainer();

        public Queue<pdfLinks> CrawlAIP(BackgroundWorker bwCrawler, string baseURL, Queue<subPages> subpages)
        {
            List<string> visitedPages = new List<string>();
            Queue<subPages> subs = new Queue<subPages>(subpages);
            Queue<pdfLinks> pdfs = new Queue<pdfLinks>();

            GetSessionID(baseURL);

            int pagesCrawled = 0;

            while (!bwCrawler.CancellationPending && subs.Count > 0)
            {
                // Remove page URL being crawled from queue
                var page = subs.Dequeue();

                // Check if page URL being crawled has been visited
                if (visitedPages.Contains(page.PageURL))
                    continue;

                // Add page URL to visited list
                visitedPages.Add(page.PageURL);

                // Create a request using a URL that can receive a post. 
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(baseURL + page.PageURL);

                //Specify CookieContainer to use
                request.CookieContainer = cookies;

                //Allow redirections for login
                request.AllowAutoRedirect = true;

                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                Stream receiveStream = response.GetResponseStream();

                Byte[] pageData;

                using (var memoryStream = new MemoryStream())
                {
                    receiveStream.CopyTo(memoryStream);
                    pageData = memoryStream.ToArray();
                }

                response.Close();

                string pageHTML = Encoding.ASCII.GetString(pageData);

                // Load page in to html document
                var doc = new HtmlAgilityPack.HtmlDocument();
                doc.LoadHtml(pageHTML);

                // Crawl page for all links
                var links = doc.DocumentNode.SelectNodes("//a[@href]");

                foreach (var link in links)
                {
                    // If link contains ".pdf", add to pdfLinks list
                    if (link.Attributes["href"].Value.Contains(".pdf"))
                    {
                        pdfs.Enqueue(new pdfLinks
                        {
                            FileName = link.InnerText,
                            FileURL = link.Attributes["href"].Value.Trim().Replace("&#xD;&#xA;", string.Empty) // Cleanse pdf link
                        });
                    }

                    // If crawling the Home page, don't look for subpages
                    if (page.PageURL != "Home.aspx")
                    {
                        // If link contains "NavWalk.aspx", add to subPages queue
                        if (link.Attributes["href"].Value.Contains("NavWalk.aspx"))
                        {
                            subs.Enqueue(new subPages
                            {
                                PageName = link.InnerText,
                                PageURL = link.Attributes["href"].Value.Replace("&amp;", "&") // Cleanse subpage link
                            }); 
                        }
                    }
                }
                pagesCrawled++;
                
                // Update status with page being crawled
                bwCrawler.ReportProgress(pagesCrawled);
            }
            return pdfs;
        }

                // Accept T&C and get ASP.NET_SessionId cookie
        public void GetSessionID(string baseURL)
        {
            string viewstate = "";
            string eventvalidation = "";
            string btnAgree = "";
            
            // Download the data to a buffer.
            Byte[] pageData;
            using (var client = new WebClient())
            {
                pageData = client.DownloadData(baseURL);
            }

            string pageHTML = Encoding.ASCII.GetString(pageData);

            // Load page in to html document
            var doc = new HtmlAgilityPack.HtmlDocument();
            doc.LoadHtml(pageHTML);

            // Crawl page for all links
            var inputs = doc.DocumentNode.SelectNodes(".//input");

            foreach (var input in inputs)
            {
                // If input name equals "__VIEWSTATE", store its value
                if (input.Attributes["name"].Value.Equals("__VIEWSTATE"))
                {
                    viewstate = HttpUtility.UrlEncode(input.Attributes["value"].Value);
                }

                // If input name equals "__EVENTVALIDATION", store its value
                if (input.Attributes["name"].Value.Equals("__EVENTVALIDATION"))
                {
                    eventvalidation = HttpUtility.UrlEncode(input.Attributes["value"].Value);
                }

                // If input name equals "__EVENTVALIDATION", store its value
                if (input.Attributes["name"].Value.Equals("btnAgree"))
                {
                    btnAgree = HttpUtility.UrlEncode(input.Attributes["value"].Value);
                }
            }

            // Create a request using a URL that can receive a post. 
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(baseURL + "Default.aspx");

            // Set the Method property of the request to POST.
            request.Method = "POST";

            // Set the ContentType property of the WebRequest.
            request.ContentType = "application/x-www-form-urlencoded";

            //Specify CookieContainer to use
            request.CookieContainer = cookies;

            // Create POST data and convert it to a byte array.
            string postData = "__VIEWSTATE=" + viewstate + "&__EVENTVALIDATION=" + eventvalidation + "&btnAgree=" + btnAgree;
            byte[] bytes = Encoding.ASCII.GetBytes(postData);

            // Set the ContentLength property of the WebRequest.
            request.ContentLength = bytes.Length;

            // Get the request stream.
            Stream dataStream = request.GetRequestStream();
            dataStream.Write(bytes, 0, bytes.Length);
            dataStream.Close();

            // Get the response.
            HttpWebResponse response = (HttpWebResponse)request.GetResponse();
        }
    }
}
